{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "path = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1  清洗数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取特征提取后的train、test数据\n",
    "train = pd.read_csv(path + '/features/train.csv', sep = '\\t')\n",
    "test = pd.read_csv(path + '/features/test.csv', sep = '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 缺失值处理\n",
    "train.fillna(-999, inplace = True)\n",
    "test.fillna(-999, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FLAG\n",
      "V1\n",
      "V2\n",
      "V3\n",
      "V4\n",
      "V5\n",
      "V6\n",
      "V7\n",
      "V8\n",
      "V9\n",
      "V10\n",
      "V11\n",
      "V12\n",
      "V13\n",
      "V14\n",
      "V15\n",
      "V16\n",
      "V17\n",
      "V18\n",
      "V19\n",
      "V20\n",
      "V21\n",
      "V22\n",
      "V23\n",
      "V24\n",
      "V25\n",
      "V26\n",
      "V27\n",
      "V28\n",
      "V29\n",
      "V30\n",
      "USER_TIM_COUNT\n",
      "USER_UNI_TIM_COUNT\n",
      "TIME_GAP_MEAN\n",
      "TIME_GAP_MAX\n",
      "TIME_GAP_MIN\n",
      "TIME_GAP_STD\n",
      "TIME_GAP_SKEW\n",
      "TIME_GAP_KURT\n",
      "ACT_DATE_MEAN\n",
      "DATE_ACT_MEAN\n",
      "DATE_REPEAT_COUNT\n",
      "ACT_COUNT_CON_DAY_MEAN\n",
      "ACT_COUNT_CON_DAY_MAX\n",
      "ACT_COUNT_CON_DAY_MIN\n",
      "ACT_COUNT_CON_DAY_SUM\n",
      "ACT_COUNT_CON_DAY_STD\n",
      "ACT_COUNT_MAX\n",
      "EVT_LBL_1_COUNT\n",
      "EVT_LBL_2_COUNT\n",
      "EVT_LBL_3_COUNT\n",
      "DAY\n",
      "IS_HIGT_ACT\n",
      "ACT_COUNT_PER_DAY\n",
      "ACT_COUNT_PER_DAY_MEAN\n",
      "ACT_COUNT_PER_DAY_MAX\n",
      "ACT_COUNT_PER_DAY_MIN\n",
      "ACT_COUNT_PER_DAY_MODE\n",
      "ACT_COUNT_PER_DAY_STD\n",
      "ACT_COUNT_PER_DAY_SKEW\n",
      "ACT_COUNT_PER_DAY_KURT\n",
      "ACT_COUNT_PER_DAY_MEDIAN\n",
      "NEXT_TIME_MAX\n",
      "NEXT_TIME_STD\n",
      "NEXT_TIME_MEAN\n",
      "NEXT_TIME_MIN\n",
      "WEEK\n",
      "ACT_COUNT_PER_WEEK\n",
      "ACT_COUNT_PER_WEEK_MEAN\n",
      "ACT_COUNT_PER_WEEK_MAX\n",
      "ACT_COUNT_PER_WEEK_MIN\n",
      "ACT_COUNT_PER_WEEK_MODE\n",
      "ACT_COUNT_PER_WEEK_STD\n",
      "ACT_COUNT_PER_WEEK_SKEW\n",
      "ACT_COUNT_PER_WEEK_KURT\n",
      "ACT_COUNT_PER_WEEK_MEDIAN\n",
      "USER_TIM_COUNT_BEFORE\n",
      "USER_UNI_TIM_COUNT_BEFORE\n",
      "TIME_GAP_MEAN_BEFORE\n",
      "TIME_GAP_MAX_BEFORE\n",
      "TIME_GAP_MIN_BEFORE\n",
      "TIME_GAP_STD_BEFORE\n",
      "TIME_GAP_SKEW_BEFORE\n",
      "TIME_GAP_KURT_BEFORE\n",
      "ACT_DATE_MEAN_BEFORE\n",
      "DATE_ACT_MEAN_BEFORE\n",
      "DATE_REPEAT_COUNT_BEFORE\n",
      "ACT_COUNT_CON_DAY_MEAN_BEFORE\n",
      "ACT_COUNT_CON_DAY_MAX_BEFORE\n",
      "ACT_COUNT_CON_DAY_MIN_BEFORE\n",
      "ACT_COUNT_CON_DAY_SUM_BEFORE\n",
      "ACT_COUNT_CON_DAY_STD_BEFORE\n",
      "ACT_COUNT_MAX_BEFORE\n",
      "EVT_LBL_1_COUNT_BEFORE\n",
      "EVT_LBL_2_COUNT_BEFORE\n",
      "EVT_LBL_3_COUNT_BEFORE\n",
      "0_x\n",
      "2_x\n",
      "0_y\n",
      "10_x\n",
      "38\n",
      "102\n",
      "139\n",
      "162\n",
      "163\n",
      "181\n",
      "257\n",
      "259\n",
      "326\n",
      "359\n",
      "372\n",
      "396\n",
      "438\n",
      "460\n",
      "508\n",
      "518\n",
      "520\n",
      "540\n",
      "604\n",
      "14_x\n",
      "15_x\n",
      "16_x\n",
      "17_x\n",
      "18_x\n",
      "19_x\n",
      "20_x\n",
      "21_x\n",
      "22_x\n",
      "23_x\n",
      "43_x\n",
      "115\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "314\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "553\n",
      "555\n",
      "557\n",
      "561_x\n",
      "569_x\n",
      "574\n",
      "575\n",
      "576\n",
      "577\n",
      "578\n",
      "701\n",
      "702\n",
      "703\n",
      "704\n",
      "705\n",
      "706\n",
      "922\n",
      "924\n",
      "1040\n",
      "1041\n",
      "1042\n",
      "1043\n",
      "1044\n",
      "1045\n",
      "1046\n",
      "1047\n",
      "1048\n",
      "1049\n",
      "1233\n",
      "1234\n",
      "1235\n",
      "1260\n",
      "1261\n",
      "1262\n",
      "1263\n",
      "1264\n",
      "1265\n",
      "1266\n",
      "1267\n",
      "1268\n",
      "1269\n",
      "1270\n",
      "1349\n",
      "1350\n",
      "1351\n",
      "1352\n",
      "1479\n",
      "1480\n",
      "1481\n",
      "1482\n",
      "1483\n",
      "1484\n",
      "1588\n",
      "1589\n",
      "1590\n",
      "1591\n",
      "1592\n",
      "1593\n",
      "1795\n",
      "1796\n",
      "1797\n",
      "1798\n",
      "1799\n",
      "1826\n",
      "1827\n",
      "1828\n",
      "1829\n",
      "1830\n",
      "1831\n",
      "1834\n",
      "1836\n",
      "1837\n",
      "1838\n",
      "1839\n",
      "1840\n",
      "1841\n",
      "1842\n",
      "1843\n",
      "1844\n",
      "1845\n",
      "1846\n",
      "1847\n",
      "1848\n",
      "1849\n",
      "1850\n",
      "1851\n",
      "1852\n",
      "1853\n",
      "1854\n",
      "1855\n",
      "1856\n",
      "1857\n",
      "1858\n",
      "1859\n",
      "1860\n",
      "1861\n",
      "1862\n",
      "1863\n",
      "1864\n",
      "1865\n",
      "1866\n",
      "1905\n",
      "1906\n",
      "1907\n",
      "1908\n",
      "1909\n",
      "1910\n",
      "1911\n",
      "1912\n",
      "1913\n",
      "1914\n",
      "1915\n",
      "1916\n",
      "2133\n",
      "2134\n",
      "2135\n",
      "2136\n",
      "2137\n",
      "2138\n",
      "2139\n",
      "2140\n",
      "2141\n",
      "2142\n",
      "2143\n",
      "2144\n",
      "2145\n",
      "2146\n",
      "2147\n",
      "2148\n",
      "2149\n",
      "2150\n",
      "2151\n",
      "2152\n",
      "2153\n",
      "2154\n",
      "2155\n",
      "2156\n",
      "2157\n",
      "2158\n",
      "2159\n",
      "2160\n",
      "2161\n",
      "2162\n",
      "2163\n",
      "2164\n",
      "2165\n",
      "2166\n",
      "2167\n",
      "14_y\n",
      "15_y\n",
      "16_y\n",
      "17_y\n",
      "18_y\n",
      "19_y\n",
      "20_y\n",
      "21_y\n",
      "22_y\n",
      "23_y\n",
      "43_y\n",
      "117\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "290\n",
      "291\n",
      "462\n",
      "558\n",
      "559\n",
      "560\n",
      "561_y\n",
      "562\n",
      "563\n",
      "564\n",
      "565\n",
      "566\n",
      "567\n",
      "568\n",
      "569_y\n",
      "570\n",
      "571\n",
      "572\n",
      "573\n",
      "889\n",
      "891\n",
      "893\n",
      "897\n",
      "905\n",
      "910\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "1196\n",
      "1197\n",
      "1198\n",
      "1199\n",
      "1200\n",
      "1201\n",
      "1523\n",
      "1525\n",
      "1677\n",
      "1678\n",
      "1679\n",
      "1680\n",
      "1681\n",
      "1682\n",
      "1683\n",
      "1684\n",
      "1685\n",
      "1686\n",
      "1687\n",
      "1688\n",
      "1689\n",
      "1690\n",
      "1691\n",
      "1692\n",
      "1693\n",
      "1694\n",
      "1695\n",
      "1696\n",
      "1697\n",
      "1698\n",
      "1699\n",
      "1700\n",
      "1701\n",
      "1702\n",
      "1703\n",
      "1704\n",
      "1705\n",
      "1706\n",
      "1707\n",
      "1708\n",
      "1709\n",
      "1710\n",
      "1711\n",
      "1712\n",
      "1713\n",
      "1714\n",
      "1715\n",
      "1716\n",
      "1717\n",
      "1718\n",
      "1719\n",
      "1720\n",
      "1722\n",
      "1723\n",
      "1724\n",
      "1725\n",
      "1726\n",
      "1727\n",
      "1728\n",
      "1729\n",
      "2003\n",
      "2004\n",
      "2005\n",
      "2031\n",
      "2032\n",
      "2033\n",
      "2034\n",
      "2035\n",
      "2036\n",
      "2037\n",
      "2039\n",
      "2040\n",
      "2041\n",
      "2042\n",
      "2043\n",
      "2044\n",
      "2045\n",
      "2046\n",
      "2047\n",
      "2048\n",
      "2049\n",
      "2050\n",
      "2051\n",
      "2052\n",
      "2053\n",
      "2054\n",
      "2055\n",
      "2056\n",
      "2057\n",
      "2058\n",
      "2059\n",
      "2061\n",
      "2062\n",
      "2063\n",
      "2064\n",
      "2065\n",
      "2066\n",
      "2067\n",
      "2068\n",
      "2069\n",
      "2070\n",
      "2071\n",
      "2072\n",
      "2073\n",
      "2074\n",
      "2075\n",
      "2076\n",
      "2077\n",
      "2078\n",
      "2079\n",
      "2080\n",
      "2204\n",
      "2205\n",
      "2206\n",
      "2207\n",
      "2637\n",
      "2638\n",
      "2639\n",
      "2640\n",
      "2641\n",
      "2642\n",
      "2643\n",
      "2644\n",
      "3121\n",
      "3122\n",
      "3123\n",
      "3124\n",
      "3125\n",
      "3126\n",
      "3127\n",
      "3128\n",
      "3129\n",
      "3130\n",
      "3131\n",
      "3132\n",
      "3133\n",
      "3134\n",
      "3135\n",
      "3136\n",
      "3137\n",
      "3138\n",
      "3139\n",
      "3140\n",
      "3141\n",
      "3142\n",
      "3143\n",
      "3144\n",
      "3145\n",
      "3146\n",
      "3147\n",
      "3148\n",
      "3149\n",
      "3150\n",
      "3151\n",
      "3152\n",
      "3153\n",
      "3154\n",
      "3155\n",
      "3156\n",
      "3157\n",
      "3158\n",
      "3159\n",
      "3160\n",
      "3161\n",
      "3162\n",
      "3164\n",
      "3166\n",
      "3167\n",
      "3168\n",
      "3169\n",
      "3170\n",
      "3171\n",
      "3172\n",
      "3173\n",
      "3174\n",
      "3179\n",
      "3180\n",
      "3181\n",
      "3182\n",
      "3183\n",
      "3184\n",
      "3185\n",
      "3186\n",
      "3187\n",
      "3188\n",
      "3189\n",
      "3190\n",
      "3191\n",
      "3198\n",
      "3199\n",
      "3202\n",
      "3207\n",
      "3210\n",
      "3211\n",
      "3216\n",
      "3221\n",
      "3226\n",
      "3227\n",
      "3228\n",
      "3231\n",
      "3234\n",
      "3241\n",
      "3243\n",
      "3245\n",
      "3246\n",
      "3247\n",
      "3248\n",
      "3492\n",
      "3493\n",
      "3494\n",
      "3495\n",
      "3496\n",
      "3497\n",
      "3498\n",
      "3612\n",
      "3613\n",
      "3614\n",
      "3615\n",
      "3616\n",
      "3617\n",
      "3618\n",
      "3619\n",
      "3620\n",
      "3621\n",
      "3622\n",
      "3623\n",
      "3626\n",
      "3627\n",
      "3628\n",
      "3640\n",
      "3641\n",
      "3642\n",
      "3643\n",
      "3644\n",
      "3645\n",
      "3646\n",
      "3647\n",
      "3648\n",
      "3649\n",
      "3651\n",
      "3652\n",
      "3653\n",
      "3654\n",
      "3657\n",
      "3662\n",
      "3663\n",
      "3664\n",
      "3665\n",
      "3666\n",
      "3667\n",
      "3670\n",
      "3671\n",
      "3672\n",
      "3673\n",
      "3674\n",
      "3676\n",
      "3680\n",
      "3682\n",
      "3683\n",
      "3684\n",
      "3685\n",
      "3687\n",
      "3689\n",
      "3691\n",
      "3692\n",
      "3693\n",
      "3695\n",
      "3697\n",
      "3700\n",
      "3701\n",
      "3702\n",
      "3706\n",
      "3707\n",
      "3711\n",
      "3712\n",
      "3714\n",
      "3722\n",
      "3723\n",
      "3724\n",
      "3725\n",
      "3726\n",
      "3727\n",
      "3728\n",
      "3729\n",
      "3730\n",
      "3731\n",
      "3732\n",
      "3734\n",
      "3735\n",
      "3736\n",
      "3737\n",
      "3738\n",
      "3739\n",
      "3740\n",
      "3741\n",
      "3742\n",
      "3743\n",
      "3744\n",
      "3745\n",
      "3746\n",
      "3747\n",
      "3748\n",
      "3749\n",
      "3751\n",
      "3752\n",
      "3753\n",
      "3754\n",
      "3755\n",
      "3756\n",
      "3757\n",
      "3758\n",
      "3759\n",
      "3760\n",
      "3761\n",
      "3762\n",
      "3763\n",
      "3764\n",
      "3765\n",
      "3766\n",
      "3767\n",
      "3768\n",
      "3769\n",
      "3770\n",
      "3771\n",
      "3772\n",
      "3773\n",
      "3774\n",
      "3775\n",
      "3776\n",
      "3777\n",
      "3778\n",
      "3779\n",
      "3780\n",
      "3781\n",
      "3782\n",
      "3783\n",
      "3784\n",
      "3785\n",
      "3786\n",
      "3787\n",
      "3788\n",
      "3789\n",
      "3790\n",
      "3791\n",
      "3792\n",
      "3793\n",
      "3794\n",
      "3795\n",
      "3796\n",
      "3797\n",
      "3798\n",
      "3800\n",
      "3801\n",
      "3802\n",
      "3803\n",
      "3804\n",
      "3805\n",
      "3806\n",
      "3807\n",
      "3808\n",
      "3809\n",
      "3810\n",
      "3811\n",
      "3812\n",
      "3813\n",
      "3814\n",
      "3815\n",
      "3816\n",
      "3817\n",
      "3818\n",
      "3819\n",
      "3820\n",
      "3821\n",
      "3825\n",
      "3826\n",
      "3827\n",
      "3828\n",
      "3829\n",
      "3830\n",
      "3831\n",
      "3832\n",
      "3833\n",
      "3834\n",
      "3835\n",
      "3836\n",
      "3837\n",
      "3838\n",
      "3839\n",
      "3840\n",
      "3841\n",
      "3842\n",
      "3843\n",
      "3844\n",
      "3845\n",
      "3846\n",
      "3847\n",
      "3848\n",
      "3849\n",
      "3850\n",
      "3851\n",
      "3852\n",
      "3853\n",
      "3854\n",
      "3855\n",
      "3856\n",
      "3857\n",
      "3858\n",
      "3859\n",
      "3860\n",
      "3861\n",
      "3862\n",
      "3863\n",
      "3864\n",
      "3865\n",
      "3866\n",
      "3867\n",
      "3868\n",
      "3869\n",
      "3870\n",
      "3871\n",
      "3872\n",
      "3873\n",
      "3920\n",
      "3921\n",
      "3922\n",
      "3923\n",
      "3924\n",
      "3925\n",
      "3926\n",
      "3927\n",
      "3928\n",
      "3929\n",
      "3930\n",
      "3931\n",
      "4270\n",
      "4271\n",
      "4272\n",
      "4273\n",
      "4275\n",
      "4276\n",
      "4277\n",
      "4278\n",
      "4279\n",
      "4280\n",
      "4281\n",
      "4282\n",
      "4283\n",
      "4284\n",
      "4285\n",
      "4286\n",
      "4287\n",
      "4288\n",
      "4289\n",
      "4290\n",
      "4291\n",
      "4292\n",
      "4293\n",
      "4294\n",
      "4295\n",
      "4296\n",
      "4297\n",
      "4298\n",
      "4299\n",
      "4300\n",
      "4301\n",
      "4302\n",
      "4303\n",
      "4304\n",
      "4305\n",
      "4307\n",
      "4308\n",
      "4309\n",
      "4310\n",
      "4311\n",
      "4312\n",
      "4313\n",
      "4314\n",
      "4315\n",
      "4316\n",
      "4317\n",
      "4318\n",
      "4319\n",
      "4320\n",
      "4321\n",
      "4322\n",
      "4323\n",
      "4324\n",
      "4325\n",
      "4326\n",
      "4327\n",
      "4328\n",
      "4329\n",
      "4330\n",
      "4331\n",
      "4332\n",
      "4333\n",
      "4334\n",
      "4335\n",
      "4336\n",
      "4337\n",
      "4338\n",
      "4339\n",
      "4344\n",
      "4345\n",
      "4346\n",
      "4347\n",
      "4348\n",
      "4349\n",
      "4350\n",
      "4351\n",
      "4352\n",
      "4353\n",
      "4354\n",
      "4355\n",
      "4356\n",
      "4357\n",
      "4358\n",
      "4359\n",
      "4360\n",
      "4361\n",
      "4362\n",
      "4363\n",
      "4364\n",
      "4365\n",
      "4366\n",
      "4367\n",
      "4368\n",
      "4369\n",
      "4370\n",
      "4371\n",
      "4372\n",
      "4373\n",
      "4374\n",
      "4375\n",
      "4376\n",
      "4377\n",
      "4378\n",
      "4380\n",
      "4381\n",
      "4382\n",
      "4383\n",
      "4384\n",
      "4386\n",
      "4387\n",
      "4388\n",
      "4389\n",
      "4390\n",
      "4391\n",
      "4392\n",
      "4393\n",
      "4394\n",
      "4395\n",
      "4396\n",
      "4397\n",
      "4398\n",
      "0\n",
      "1_x\n",
      "2_y\n",
      "3_x\n",
      "4_x\n",
      "5_x\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10_y\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "1_y\n",
      "2\n",
      "3_y\n",
      "4_y\n",
      "5_y\n"
     ]
    }
   ],
   "source": [
    "# 所有列名\n",
    "features = list(train.columns)\n",
    "features.pop(0)\n",
    "for i in features:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2  探索特征数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入统计分析模块\n",
    "from scipy import stats\n",
    "\n",
    "# 察看各变量间的P值与拟合情况\n",
    "statistic = []\n",
    "for i in range(len(features) - 1):\n",
    "    stat = []\n",
    "    for j in range(i + 1, len(features)):\n",
    "        # 计算两变量间的相关系数、P值\n",
    "        r, p = stats.pearsonr(train[features[i]], train[features[j]])\n",
    "        stat.append(features[i])\n",
    "        stat.append(features[j])\n",
    "        stat.append(r)\n",
    "        stat.append(p)\n",
    "    statistic.append(stat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_stats = pd.DataFrame(statistic,\n",
    "                             columns = ['FEATURE_X',\n",
    "                                        'FEATURE_Y',\n",
    "                                        'PERSON_R',\n",
    "                                        'P']\n",
    "                            )\n",
    "del statistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_stats = feature_stats.groupby(['FEATURE_X'])\n",
    "feature_stats = feature_stats.sort_values(['P', 'PERSON_R'], ascending = [True, False])\n",
    "feature_stats.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_stats.to_csv(path + '/features/feature_stats.csv', index = None, sep = '\\t')\n",
    "del feature_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 相关性最大的前100个特征间的相关系数显示\n",
    "corr = train[features].corr()\n",
    "feature_nums =  101\n",
    "cols =  corr.nlargest(feature_nums, 'FLAG')['FLAG'].index\n",
    "cm = np.corrcoef(train[cols].values.T)\n",
    "\n",
    "f, ax = plt.subplots(figsize = (100, 100))\n",
    "sns.set(font_scale = 1.25)\n",
    "hm = sns.heatmap(cm,\n",
    "                 cbar = True,\n",
    "                 annot = True,\n",
    "                 square = True,\n",
    "                 fmt = '.2f',\n",
    "                 annot_kws = {'size' : 8},\n",
    "                 yticklabels = cols.values,\n",
    "                 xticklabels = cols.values\n",
    "                )\n",
    "\n",
    "# 设置刻度字体大小  \n",
    "plt.xticks(fontsize = 12)  \n",
    "plt.yticks(fontsize = 12)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (8, 6))\n",
    "for i in range(1, 96):\n",
    "    data_sub = pd.concat([train['FLAG'], train[features[j]]], axis = 1)\n",
    "    sns.set_palette('muted')\n",
    "    sns.jointplot(x = 'FLAG',\n",
    "                  y = features[j],\n",
    "                  data = data_sub,\n",
    "                  kind = 'kde',\n",
    "                  color = 'b',\n",
    "                  space = 0.5,\n",
    "                  ratio = 5)\n",
    "    plt.title(features[i] + ' - ' + features[j], fontsize = 18)\n",
    "    plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签  \n",
    "    plt.rcParams['axes.unicode_minus'] = False   # 用来正常显示负号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 先初步划分数据集\n",
    "train_x = train.drop(['USRID', 'FLAG', 'DAY'], axis = 1).values\n",
    "train_y = train['FLAG'].values\n",
    "del train\n",
    "\n",
    "test_x = test.drop(['USRID', 'DAY'], axis = 1).values\n",
    "UID = test[['USRID']]\n",
    "del test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "UID.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 3  XGBoost建模预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import operator\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.1 保留所有特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.1.1 将训练集划分为X_dtrain, X_deval, y_dtrain, y_deval，训练xgboost模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, learning_curve\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 所有列名\n",
    "features = list(train_x.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 平衡数据处理\n",
    "randomState = 10\n",
    "np.random.seed(randomState)\n",
    "\n",
    "X_dtrain, X_deval, y_dtrain, y_deval = train_test_split(train_x,\n",
    "                                                        train_y,\n",
    "                                                        test_size = 0.3,\n",
    "                                                        random_state = randomState)\n",
    "\n",
    "weights = ((train_y == 0).sum() / (1.0 * (train_y == 1).sum()))\n",
    "print('平衡数据调整权重：', weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 网格搜索\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "# 需搜索的参数\n",
    "tree_param_grid = {'n_estimators': list(range(300, 1100, 100)),      # 树的个数\n",
    "                   'max_depth': list(range(3, 10))                   # 树的深度，越大越容易过拟合\n",
    "                  }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(X_dtrain, y_dtrain.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'min_child_weight': list(range(1, 10))            # min_child_weight = 1\n",
    "                                                                     # 即叶子节点中最少需要包含 100 个样本\n",
    "                                                                     # 参数值越小，越容易 overfitting\n",
    "                   }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(X_dtrain, y_dtrain.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'subsample': list(np.linspace(0.5, 1, 6)),        # 每棵树随机采样的比例，通常0.5~1 \n",
    "                   'colsample_bytree': list(np.linspace(0.5, 1, 6))  # 生成树时进行的特征采样占比，通常0.5~1\n",
    "                   }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(X_dtrain, y_dtrain.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'reg_alpha': list(range(1, 10)),                  # 权重值的L1正则化项参数\n",
    "                   'reg_lambda': list(range(1, 10))                  # 权重值的L2正则化项参数\n",
    "                  }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(X_dtrain, y_dtrain.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'seed': list(range(100, 1001, 100))               # 随机种子\n",
    "                  }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(X_dtrain, y_dtrain.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'seed': list(range(100, 1001, 100))               # 随机种子\n",
    "                  }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(X_dtrain, y_dtrain.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练XGBoost模型\n",
    "model1 = xgb.XGBClassifier()\n",
    "y_pred = model1.fit(X_dtrain, y_dtrain).predict(X_deval)\n",
    "print('Accuracy : ', accuracy_score(y_deval, y_pred))\n",
    "del y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 确定特征因子重要性\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 绘制重要性前 100 的特征重要性评分\n",
    "colours = plt.cm.Set1(np.linspace(0, 1, 100))\n",
    "fig = plt.figure(figsize = (16, 24))\n",
    "ax = fig.add_subplot(111)\n",
    "ax = xgb.plot_importance(model1,\n",
    "                         max_number_features = 100,\n",
    "                         height = 0.8,\n",
    "                         color = colours,\n",
    "                         grid = False,\n",
    "                         show_values = True,\n",
    "                         importance_type = 'cover',\n",
    "                         ax = ax)\n",
    "for axis in ['top','bottom','left','right']:\n",
    "    ax.spines[axis].set_linewidth(2)\n",
    "\n",
    "ax.set_xlabel('重要性评分', size = 16)\n",
    "ax.set_ylabel('特征', size = 16)\n",
    "ax.set_yticklabels(ax.get_yticklabels(), size = 12)\n",
    "ax.set_title('模型学习后特征重要性排序', size = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型重要性的决策树可视化\n",
    "os.environ['PATH'] += os.pathsep + 'D:/Graphviz2.38/bin'\n",
    "xgb.to_graphviz(model1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 学习曲线显示精度\n",
    "trainSizes, trainScores, crossValScores = learning_curve(\n",
    "    XGBClassifier(),\n",
    "    X_dtrain,\n",
    "    y_dtrain,\n",
    "    cv = 5,\n",
    "    scoring = 'accuracy'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainScoresMean = np.mean(trainScores, axis = 1)\n",
    "trainScoresStd = np.std(trainScores, axis = 1)\n",
    "crossValScoresMean = np.mean(crossValScores, axis = 1)\n",
    "crossValScoresStd = np.std(crossValScores, axis = 1)\n",
    "\n",
    "fig = plt.figure(figsize = (12, 8))\n",
    "plt.fill_between(trainSizes,\n",
    "                 trainScoresMean - trainScoresStd,\n",
    "                 trainScoresMean + trainScoresStd,\n",
    "                 alpha = 0.1,\n",
    "                 color = 'b')\n",
    "plt.fill_between(trainSizes,\n",
    "                 crossValScoresMean - crossValScoresStd,\n",
    "                 crossValScoresMean + crossValScoresStd,\n",
    "                 alpha = 0.1,\n",
    "                 color = 'r')\n",
    "plt.plot(trainSizes,\n",
    "         trainScores.mean(axis = 1),\n",
    "         'o-',\n",
    "         label = 'TRAIN',\n",
    "         color = 'b')\n",
    "plt.plot(trainSizes,\n",
    "         crossValScores.mean(axis = 1),\n",
    "         'o-',\n",
    "         label = 'CROSS-VAL',\n",
    "         color = 'r')\n",
    "\n",
    "ax = plt.gca()\n",
    "for axis in ['top', 'bottom', 'left', 'right']:\n",
    "    ax.spines[axis].set_linewidth(2)\n",
    "\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "plt.legend(handles,\n",
    "           ['TRAIN', 'CROSS-VAL'],\n",
    "           bbox_to_anchor = (0.8, 0.15),\n",
    "           loc = 2,\n",
    "           borderaxespad = 0,\n",
    "           fontsize = 16)\n",
    "plt.xlabel('TRAIN', size = 16)\n",
    "plt.ylabel('AUC', size = 16)\n",
    "plt.title('Learning Curves (all features and part samples)', size = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X_dtrain\n",
    "del X_deval\n",
    "del y_dtrain\n",
    "del y_deval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输出结果\n",
    "res_pred = model1.predict(test_x)\n",
    "a = pd.DataFrame(res_pred, columns = ['RST'])\n",
    "res = pd.concat([UID, a['RST']], axis = 1)\n",
    "res.to_csv(path + '/result/all_feat_split_result.csv', index = None, sep = '\\t')\n",
    "del res_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.1.2 用所有训练集训练xgboost模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 需搜索最优超参数\n",
    "tree_param_grid = {'n_estimators': list(range(300, 1100, 100)),      # 树的个数\n",
    "                   'max_depth': list(range(3, 10))                   # 树的深度，越大越容易过拟合\n",
    "                  }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(train_x, train_y.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'min_child_weight': list(range(1, 10))            # min_child_weight = 1\n",
    "                                                                     # 即叶子节点中最少需要包含 100 个样本\n",
    "                                                                     # 参数值越小，越容易 overfitting\n",
    "                   }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(train_x, train_y.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'subsample': list(np.linspace(0.5, 1, 6)),        # 每棵树随机采样的比例，通常0.5~1 \n",
    "                   'colsample_bytree': list(np.linspace(0.5, 1, 6))  # 生成树时进行的特征采样占比，通常0.5~1\n",
    "                   }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(train_x, train_y.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'reg_alpha': list(range(1, 10)),                  # 权重值的L1正则化项参数\n",
    "                   'reg_lambda': list(range(1, 10))                  # 权重值的L2正则化项参数\n",
    "                  }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(train_x, train_y.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_param_grid = {'seed': list(range(100, 1001, 100))               # 随机种子\n",
    "                  }\n",
    "gsearch = GridSearchCV(\n",
    "    estimator = xgb.XGBClassifier(silent = 1,\n",
    "                                  scale_pos_weight = weights,\n",
    "                                  n_jobs = -1,\n",
    "                                  eval_metric = 'auc'\n",
    "                                 ),\n",
    "    param_grid = tree_param_grid,\n",
    "    cv = 5,\n",
    "    n_jobs = -1)\n",
    "\n",
    "gsearch.fit(train_x, train_y.ravel())\n",
    "\n",
    "# 打印每个搜索的分数\n",
    "print('grid scores :\\n')\n",
    "for i in gsearch.grid_scores_:\n",
    "    print(i)\n",
    "# 打印最佳参数\n",
    "print('\\nbest params : ', gsearch.best_params_)\n",
    "# 打印最高分数\n",
    "print('\\nbest score : ', gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练XGBoost模型\n",
    "model2 = xgb.XGBClassifier()\n",
    "model2.fit(train_x, train_y)\n",
    "# 输出结果\n",
    "res_pred = model2.predict(test_x)\n",
    "a = pd.DataFrame(res_pred, columns = ['RST'])\n",
    "res = pd.concat([UID, a['RST']], axis = 1)\n",
    "res.to_csv(path + '/result/all_feat_result.csv', index = None, sep = '\\t')\n",
    "del res_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型重要性的决策树可视化\n",
    "os.environ['PATH'] += os.pathsep + 'D:/Graphviz2.38/bin'\n",
    "xgb.to_graphviz(model1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 学习曲线显示精度\n",
    "trainSizes, trainScores, crossValScores = learning_curve(\n",
    "    XGBClassifier(),\n",
    "    train_x,\n",
    "    train_y,\n",
    "    cv = 5,\n",
    "    scoring = 'accuracy'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainScoresMean = np.mean(trainScores, axis = 1)\n",
    "trainScoresStd = np.std(trainScores, axis = 1)\n",
    "crossValScoresMean = np.mean(crossValScores, axis = 1)\n",
    "crossValScoresStd = np.std(crossValScores, axis = 1)\n",
    "\n",
    "fig = plt.figure(figsize = (12, 8))\n",
    "plt.fill_between(trainSizes,\n",
    "                 trainScoresMean - trainScoresStd,\n",
    "                 trainScoresMean + trainScoresStd,\n",
    "                 alpha = 0.1,\n",
    "                 color = 'b')\n",
    "plt.fill_between(trainSizes,\n",
    "                 crossValScoresMean - crossValScoresStd,\n",
    "                 crossValScoresMean + crossValScoresStd,\n",
    "                 alpha = 0.1,\n",
    "                 color = 'r')\n",
    "plt.plot(trainSizes,\n",
    "         trainScores.mean(axis = 1),\n",
    "         'o-',\n",
    "         label = 'TRAIN',\n",
    "         color = 'b')\n",
    "plt.plot(trainSizes,\n",
    "         crossValScores.mean(axis = 1),\n",
    "         'o-',\n",
    "         label = 'CROSS-VAL',\n",
    "         color = 'r')\n",
    "\n",
    "ax = plt.gca()\n",
    "for axis in ['top', 'bottom', 'left', 'right']:\n",
    "    ax.spines[axis].set_linewidth(2)\n",
    "\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "plt.legend(handles,\n",
    "           ['TRAIN', 'CROSS-VAL'],\n",
    "           bbox_to_anchor = (0.8, 0.15),\n",
    "           loc = 2,\n",
    "           borderaxespad = 0,\n",
    "           fontsize = 16)\n",
    "plt.xlabel('TRAIN', size = 16)\n",
    "plt.ylabel('AUC', size = 16)\n",
    "plt.title('Learning Curves (all features and all samples)', size = 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.2 根据获得的统计数据feature_stats再提取特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.2.1 将训练集划分为X_dtrain, X_deval, y_dtrain, y_deval，训练xgboost模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.2.2 用所有训练集训练xgboost模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.3 利用开源已调参的模型预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 转化为xgboost需要格式\n",
    "xgb_train = xgb.DMatrix(train_x, label = train_y)\n",
    "xgb_test = xgb.DMatrix(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型参数\n",
    "params = {'booster': 'gbtree',\n",
    "          'objective': 'rank:pairwise',  # 二分类的问题\n",
    "          # 'gamma':0.1,                 # 用于控制是否后剪枝的参数,越大越保守，一般0.1、0.2\n",
    "          'max_depth': 5,                # 构建树的深度，越大越容易过拟合\n",
    "          # 'lambda':2,                  # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。\n",
    "          'subsample': 0.7,              # 随机采样训练样本\n",
    "          'colsample_bytree': 0.7,       # 生成树时进行的列采样\n",
    "          'min_child_weight': 3,         # 这个参数默认是 1，即每个叶子里面 h 的和至少是多少\n",
    "                                         # 对正负样本不均衡时的 0-1 分类而言, 假设 h 在 0.01 附近\n",
    "                                         # min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本\n",
    "                                         # 这个参数非常影响结果，控制叶子节点中二阶导的和的最小值\n",
    "                                         # 该参数值越小，越容易 overfitting\n",
    "          'silent': 0,                   # 设置成1则没有运行信息输出，最好是设置为0\n",
    "          'eta': 0.03,                   # 如同学习率\n",
    "          'eval_metric': 'auc'           # 评价方式\n",
    "         }\n",
    "\n",
    "plst = list(params.items())\n",
    "num_rounds = 500  # 迭代次数\n",
    "watchlist = [(xgb_train, 'train')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练模型并预测\n",
    "model1 = xgb.train(plst, xgb_train, num_rounds, watchlist)\n",
    "xgb_pred = model1.predict(xgb_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.DataFrame(pred_value, columns = ['RST'])\n",
    "res = pd.concat([UID, a['RST']], axis = 1)\n",
    "res.to_csv(path + '/result/exist_result.csv', index = None, sep = '\\t')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
